
<!DOCTYPE HTML>
<html lang="" >
    <head>
        <meta charset="UTF-8">
        <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
        <title>Flume基础 · GitBook</title>
        <meta http-equiv="X-UA-Compatible" content="IE=edge" />
        <meta name="description" content="">
        <meta name="generator" content="GitBook 3.2.3">
        
        
        
    
    <link rel="stylesheet" href="../gitbook/style.css">

    
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
                
            
        

    

    
        
    
        
    
        
    
        
    
        
    
        
    

        
    
    
    <meta name="HandheldFriendly" content="true"/>
    <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
    <meta name="apple-mobile-web-app-capable" content="yes">
    <meta name="apple-mobile-web-app-status-bar-style" content="black">
    <link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
    <link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">

    
    

    </head>
    <body>
        
<div class="book">
    <div class="book-summary">
        
            
<div id="book-search-input" role="search">
    <input type="text" placeholder="Type to search" />
</div>

            
                <nav role="navigation">
                


<ul class="summary">
    
    

    

    
        
        
    
        <li class="chapter " data-level="1.1" data-path="../">
            
                <a href="../">
            
                    
                    简介
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2" >
            
                <span>
            
                    
                    Hadoop总结
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.2.1" data-path="../chapter1/hadoopBase.html">
            
                <a href="../chapter1/hadoopBase.html">
            
                    
                    Hadoop基础
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.2" data-path="../chapter1/Hadoop性能优化.html">
            
                <a href="../chapter1/Hadoop性能优化.html">
            
                    
                    Hadoop性能优化
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.3" data-path="../chapter1/Hadoop故障处理.html">
            
                <a href="../chapter1/Hadoop故障处理.html">
            
                    
                    Hadoop故障处理
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.3" >
            
                <span>
            
                    
                    Flume总结
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter active" data-level="1.3.1" data-path="FlumeBase.html">
            
                <a href="FlumeBase.html">
            
                    
                    Flume基础
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.4" >
            
                <span>
            
                    
                    Hive总结
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.4.1" data-path="../chapter4/HiveBase.html">
            
                <a href="../chapter4/HiveBase.html">
            
                    
                    Hive基础
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.5" >
            
                <span>
            
                    
                    数仓总结
            
                </span>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.5.1" data-path="../chapter5/数仓基础.html">
            
                <a href="../chapter5/数仓基础.html">
            
                    
                    数仓理论
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.5.2" data-path="../chapter5/数仓面试题.html">
            
                <a href="../chapter5/数仓面试题.html">
            
                    
                    题目汇总]
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    

    

    <li class="divider"></li>

    <li>
        <a href="https://www.gitbook.com" target="blank" class="gitbook-link">
            Published with GitBook
        </a>
    </li>
</ul>


                </nav>
            
        
    </div>

    <div class="book-body">
        
            <div class="body-inner">
                
                    

<div class="book-header" role="navigation">
    

    <!-- Title -->
    <h1>
        <i class="fa fa-circle-o-notch fa-spin"></i>
        <a href=".." >Flume基础</a>
    </h1>
</div>




                    <div class="page-wrapper" tabindex="-1" role="main">
                        <div class="page-inner">
                            
<div id="book-search-results">
    <div class="search-noresults">
    
                                <section class="normal markdown-section">
                                
                                <h1 id="flume-&#x603B;&#x7ED3;">Flume &#x603B;&#x7ED3;</h1>
<h2 id="flume-&#x7EC4;&#x6210;&#x3001;put&#x4E8B;&#x7269;&#xFF0C;take&#x4E8B;&#x7269;">Flume &#x7EC4;&#x6210;&#x3001;Put&#x4E8B;&#x7269;&#xFF0C;Take&#x4E8B;&#x7269;</h2>
<ul>
<li>Taildir Source&#xFF1A;&#x65AD;&#x70B9;&#x7EED;&#x4F20;&#x3001;&#x591A;&#x76EE;&#x5F55;&#x3002;Flume1.6 &#x4EE5;&#x524D;&#x9700;&#x8981;&#x81EA;&#x5DF1;&#x81EA;&#x5B9A;&#x4E49; Source &#x8BB0;&#x5F55;-&#x6BCF;&#x6B21;&#x8BFB;&#x53D6;&#x6587;&#x4EF6;&#x4F4D;&#x7F6E;&#xFF0C;&#x5B9E;&#x73B0;&#x65AD;&#x70B9;&#x7EED;&#x4F20;&#x3002;</li>
<li>File Channel&#xFF1A;&#x6570;&#x636E;&#x5B58;&#x50A8;&#x5728;&#x78C1;&#x76D8;&#xFF0C;&#x5B95;&#x673A;&#x6570;&#x636E;&#x53EF;&#x4EE5;&#x4FDD;&#x5B58;&#x3002;&#x4F46;&#x662F;&#x4F20;&#x8F93;&#x901F;&#x7387;&#x6162;&#x3002;&#x9002;&#x5408;&#x5BF9;&#x6570;&#x636E;-&#x4F20;&#x8F93;&#x53EF;&#x9760;&#x6027;&#x8981;&#x6C42;&#x9AD8;&#x7684;&#x573A;&#x666F;&#xFF0C;&#x6BD4;&#x5982;&#xFF0C;&#x91D1;&#x878D;&#x884C;&#x4E1A;&#x3002;</li>
<li>Memory Channel&#xFF1A;&#x6570;&#x636E;&#x5B58;&#x50A8;&#x5728;&#x5185;&#x5B58;&#x4E2D;&#xFF0C;&#x5B95;&#x673A;&#x6570;&#x636E;&#x4E22;&#x5931;&#x3002;&#x4F20;&#x8F93;&#x901F;&#x7387;&#x5FEB;&#x3002;&#x9002;&#x5408;&#x5BF9;&#x6570;&#x636E;&#x4F20;&#x8F93;&#x53EF;&#x9760;&#x6027;&#x8981;&#x6C42;&#x4E0D;&#x9AD8;&#x7684;&#x573A;&#x666F;&#xFF0C;&#x6BD4;&#x5982;&#xFF0C;&#x666E;&#x901A;&#x7684;&#x65E5;&#x5FD7;&#x6570;&#x636E;&#x3002;</li>
<li>Kafka Channel&#xFF1A;&#x51CF;&#x5C11;&#x4E86; Flume &#x7684; Sink &#x9636;&#x6BB5;&#xFF0C;&#x63D0;&#x9AD8;&#x4E86;&#x4F20;&#x8F93;&#x6548;&#x7387;&#x3002; </li>
<li>Source &#x5230; Channel &#x662F; Put &#x4E8B;&#x52A1;</li>
<li>Channel &#x5230; Sink &#x662F; Take &#x4E8B;&#x52A1;</li>
</ul>
<p><strong>put&#x4E8B;&#x52A1;&#x6B65;&#x9AA4;&#xFF1A;</strong></p>
<ul>
<li>doput :&#x5148;&#x5C06;&#x6279;&#x6570;&#x636E;&#x5199;&#x5165;&#x4E34;&#x65F6;&#x7F13;&#x51B2;&#x533A;putlist&#x91CC;&#x9762;</li>
<li>docommit&#xFF1A;&#x53BB;&#x68C0;&#x67E5;channel&#x91CC;&#x9762;&#x6709;&#x6CA1;&#x6709;&#x7A7A;&#x4F4D;&#x7F6E;&#xFF0C;&#x5982;&#x679C;&#x6709;&#x5C31;&#x4F20;&#x5165;&#x6570;&#x636E;&#xFF0C;&#x5982;&#x679C;&#x6CA1;&#x6709;&#x90A3;&#x4E48;dorollback&#x5C31;&#x628A;&#x6570;&#x636E;&#x56DE;&#x6EDA;&#x5230;putlist&#x91CC;&#x9762;&#x3002;</li>
</ul>
<p><strong>take&#x4E8B;&#x52A1;&#x6B65;&#x9AA4;&#xFF1A;</strong></p>
<p>= dotake&#xFF1A;&#x5C06;&#x6570;&#x636E;&#x8BFB;&#x53D6;&#x5230;&#x4E34;&#x65F6;&#x7F13;&#x51B2;&#x533A;takelist&#xFF0C;&#x5E76;&#x5C06;&#x6570;&#x636E;&#x4F20;&#x5230;hdfs&#x4E0A;&#x3002;</p>
<ul>
<li>docommit :&#x53BB;&#x5224;&#x65AD;&#x6570;&#x636E;&#x53D1;&#x9001;&#x662F;&#x5426;&#x6210;&#x529F;&#xFF0C;&#x82E5;&#x6210;&#x529F;&#x90A3;&#x4E48;&#x6E05;&#x9664;&#x4E34;&#x65F6;&#x7F13;&#x51B2;&#x533A;takelist
&#x82E5;&#x4E0D;&#x6210;&#x529F;&#xFF08;&#x6BD4;&#x5982;hdfs&#x7CFB;&#x7EDF;&#x670D;&#x52A1;&#x5668;&#x5D29;&#x6E83;&#x7B49;&#xFF09;&#x90A3;&#x4E48;dorollback&#x5C06;&#x6570;&#x636E;&#x56DE;&#x6EDA;&#x5230;channel&#x91CC;&#x9762;&#x3002;</li>
</ul>
<h2 id="flume-&#x62E6;&#x622A;&#x5668;">Flume &#x62E6;&#x622A;&#x5668;</h2>
<p> <strong>&#x62E6;&#x622A;&#x5668;&#x6CE8;&#x610F;&#x4E8B;&#x9879;</strong> </p>
<ul>
<li>&#x9879;&#x76EE;&#x4E2D;&#x81EA;&#x5B9A;&#x4E49;&#x4E86;&#xFF1A;ETL &#x62E6;&#x622A;&#x5668;&#x548C;&#x533A;&#x5206;&#x7C7B;&#x578B;&#x62E6;&#x622A;&#x5668;&#x3002;</li>
<li>&#x91C7;&#x7528;&#x4E24;&#x4E2A;&#x62E6;&#x622A;&#x5668;&#x7684;&#x4F18;&#x7F3A;&#x70B9;&#xFF1A;&#x4F18;&#x70B9;&#xFF0C;&#x6A21;&#x5757;&#x5316;&#x5F00;&#x53D1;&#x548C;&#x53EF;&#x79FB;&#x690D;&#x6027;&#xFF1B;&#x7F3A;&#x70B9;&#xFF0C;&#x6027;&#x80FD;&#x4F1A;&#x4F4E;&#x4E00;&#x4E9B;</li>
</ul>
<p><strong>&#x81EA;&#x5B9A;&#x4E49;&#x62E6;&#x622A;&#x5668;&#x6B65;&#x9AA4;</strong></p>
<ol>
<li>&#x5B9E;&#x73B0; Interceptor</li>
<li>&#x91CD;&#x5199;&#x56DB;&#x4E2A;&#x65B9;&#x6CD5;<ol>
<li>initialize &#x521D;&#x59CB;&#x5316;</li>
<li>public Event intercept(Event event) &#x5904;&#x7406;&#x5355;&#x4E2A; Event</li>
<li>public Event intercept(Event event) &#x5904;&#x7406;&#x5355;&#x4E2A; Event</li>
<li>close &#x65B9;&#x6CD5;</li>
</ol>
</li>
</ol>
<h2 id="hdfs-sink-&#x5C0F;&#x6587;&#x4EF6;&#x5904;&#x7406;">HDFS Sink &#x5C0F;&#x6587;&#x4EF6;&#x5904;&#x7406;</h2>
<p>&#x5B98;&#x65B9;&#x9ED8;&#x8BA4;&#x7684;&#x8FD9;&#x4E09;&#x4E2A;&#x53C2;&#x6570;&#x914D;&#x7F6E;&#x5199;&#x5165; HDFS &#x540E;&#x4F1A;&#x4EA7;&#x751F;&#x5C0F;&#x6587;&#x4EF6;&#xFF0C;hdfs.rollInterval&#x3001;hdfs.rollSize&#x3001;
hdfs.rollCount
&#x57FA; &#x4E8E; &#x4EE5; &#x4E0A; hdfs.rollInterval=3600 &#xFF0C; hdfs.rollSize=134217728 &#xFF0C; hdfs.rollCount =0 &#xFF0C;
hdfs.roundValue=3600&#xFF0C;hdfs.roundUnit= second &#x51E0;&#x4E2A;&#x53C2;&#x6570;&#x7EFC;&#x5408;&#x4F5C;&#x7528;&#xFF0C;&#x6548;&#x679C;&#x5982;&#x4E0B;&#xFF1A;</p>
<ol>
<li>tmp &#x6587;&#x4EF6;&#x5728;&#x8FBE;&#x5230; 128M &#x65F6;&#x4F1A;&#x6EDA;&#x52A8;&#x751F;&#x6210;&#x6B63;&#x5F0F;&#x6587;&#x4EF6;</li>
<li>tmp &#x6587;&#x4EF6;&#x521B;&#x5EFA;&#x8D85; 3600 &#x79D2;&#x65F6;&#x4F1A;&#x6EDA;&#x52A8;&#x751F;&#x6210;&#x6B63;&#x5F0F;&#x6587;&#x4EF6;</li>
</ol>
<p>&#x4E3E;&#x4F8B;&#xFF1A;
&#x5728; <code>2018-01-01 05:23</code> &#x7684;&#x65F6;&#x4FAF; <code>sink</code> &#x63A5;&#x6536;&#x5230;&#x6570;&#x636E;&#xFF0C;&#x90A3;&#x4F1A;&#x4EA7;&#x751F;&#x5982;&#x4E0B; <code>tmp</code> &#x6587;&#x4EF6;&#xFF1A;
<code>/atguigu/20180101/atguigu.201801010520.tmp</code></p>
<p>&#x5373;&#x4F7F;&#x6587;&#x4EF6;&#x5185;&#x5BB9;&#x6CA1;&#x6709;&#x8FBE;&#x5230; 128M&#xFF0C;&#x4E5F;&#x4F1A;&#x5728; 06:23 &#x65F6;&#x6EDA;&#x52A8;&#x751F;&#x6210;&#x6B63;&#x5F0F;&#x6587;&#x4EF6;&#x3002;</p>

                                
                                </section>
                            
    </div>
    <div class="search-results">
        <div class="has-results">
            
            <h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
            <ul class="search-results-list"></ul>
            
        </div>
        <div class="no-results">
            
            <h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
            
        </div>
    </div>
</div>

                        </div>
                    </div>
                
            </div>

            
                
                
            
        
    </div>

    <script>
        var gitbook = gitbook || [];
        gitbook.push(function() {
            gitbook.page.hasChanged({"page":{"title":"Flume基础","level":"1.3.1","depth":2,"next":{"title":"Hive总结","level":"1.4","depth":1,"ref":"","articles":[{"title":"Hive基础","level":"1.4.1","depth":2,"path":"chapter4/HiveBase.md","ref":"chapter4/HiveBase.md","articles":[]}]},"previous":{"title":"Flume总结","level":"1.3","depth":1,"ref":"","articles":[{"title":"Flume基础","level":"1.3.1","depth":2,"path":"chapter2/FlumeBase.md","ref":"chapter2/FlumeBase.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"chapter2/FlumeBase.md","mtime":"2021-03-29T03:10:19.038Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2021-03-30T00:41:16.128Z"},"basePath":"..","book":{"language":""}});
        });
    </script>
</div>

        
    <script src="../gitbook/gitbook.js"></script>
    <script src="../gitbook/theme.js"></script>
    
        
        <script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-search/search.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
        
    

    </body>
</html>

